{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_space(string, threshold = 0.3):\n",
    "    string = [s for s in string if not (s == ' ' and random.random() >= threshold)]\n",
    "    return ''.join(string)\n",
    "\n",
    "def package(string, repeat = 2):\n",
    "    result = [(string, string)]\n",
    "    result.append((string.lower(), string.lower()))\n",
    "    \n",
    "    for _ in range(repeat):\n",
    "        result.append((remove_space(string), string))\n",
    "        result.append((remove_space(string.lower()), string.lower()))\n",
    "        \n",
    "    return result\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "def loop(strings):\n",
    "    results = []\n",
    "    for i in tqdm(range(len(strings))):\n",
    "        p = package(strings[i])\n",
    "        results.extend(p)\n",
    "    return results\n",
    "\n",
    "def slide(strings, n = 5):\n",
    "    result = []\n",
    "    for i in range(0, len(strings), len(strings) - (n - 1)):\n",
    "        result.append(strings[i: i + n])\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = ['/home/husein/pure-text/filtered-dumping-wiki.txt',\n",
    "        '/home/husein/pure-text/dumping-cleaned-news.txt',\n",
    "        '/home/husein/pure-text/dumping-iium.txt']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "363578"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(files[0]) as fopen:\n",
    "    data = fopen.read().split('\\n')\n",
    "    \n",
    "results, result = [], []\n",
    "for i in data:\n",
    "    if len(i) and i[-1] != '.':\n",
    "        i = i + '.'\n",
    "    if not len(i) and len(result):\n",
    "        results.append(result)\n",
    "        result = []\n",
    "    else:\n",
    "        if len(i):\n",
    "            result.append(i)\n",
    "        \n",
    "if len(result):\n",
    "    results.append(result)\n",
    "    \n",
    "len(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "def loop(strings):\n",
    "    results = []\n",
    "    for i in tqdm(range(len(strings))):\n",
    "        try:\n",
    "            slided = slide(strings[i])\n",
    "            slided = [s for s in slided if len(s) > 1]\n",
    "            for s in slided:\n",
    "                s = ' '.join(s)\n",
    "                p = package(s)\n",
    "                results.extend(p)\n",
    "        except:\n",
    "            pass\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3125/3125 [00:00<00:00, 8116.83it/s]]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 7635.77it/s] \n",
      "100%|██████████| 3125/3125 [00:00<00:00, 6508.34it/s]\n",
      " 97%|█████████▋| 3042/3125 [00:00<00:00, 7539.97it/s]\n",
      " 93%|█████████▎| 2897/3125 [00:00<00:00, 5373.00it/s]\n",
      "\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 5784.02it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 6140.78it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 6661.56it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 7280.02it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 7287.13it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 7619.61it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 6370.47it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 6937.67it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 5536.23it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 5647.04it/s]\n"
     ]
    }
   ],
   "source": [
    "import cleaning\n",
    "\n",
    "results1 = cleaning.multiprocessing(random.sample(results, 50000), loop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cleaning_row(string):\n",
    "    string = string.replace('\\n', ' ').replace('\\t', ' ')\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip()\n",
    "    return string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 197802/197802 [00:12<00:00, 16179.82it/s]\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "with tf.io.gfile.GFile('segmentation-multisentences-wiki.tsv', \"w\") as outfile:\n",
    "    for i in tqdm(range(len(results1))):\n",
    "        l = cleaning_row(results1[i][0])\n",
    "        r = cleaning_row(results1[i][1])\n",
    "        outfile.write(\"%s\\t%s\\n\" % (l, r))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3656919"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(files[1]) as fopen:\n",
    "    data = fopen.read().split('\\n')\n",
    "    \n",
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "results, result = [], []\n",
    "for i in data:\n",
    "    if len(i) and i[-1] != '.':\n",
    "        i = i + '.'\n",
    "    if not len(i) and len(result):\n",
    "        results.append(result)\n",
    "        result = []\n",
    "    else:\n",
    "        if len(i):\n",
    "            result.append(i)\n",
    "        \n",
    "if len(result):\n",
    "    results.append(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1875/1875 [00:00<00:00, 2226.89it/s]\n",
      "100%|██████████| 1875/1875 [00:01<00:00, 1788.50it/s]\n",
      "100%|██████████| 1875/1875 [00:01<00:00, 1852.38it/s]\n",
      "100%|██████████| 1875/1875 [00:01<00:00, 1637.18it/s]\n",
      "100%|██████████| 1875/1875 [00:01<00:00, 1658.73it/s]\n",
      "100%|██████████| 1875/1875 [00:01<00:00, 1803.00it/s]\n",
      "100%|██████████| 1875/1875 [00:01<00:00, 1683.21it/s]\n",
      "100%|██████████| 1875/1875 [00:01<00:00, 1812.66it/s]\n",
      "100%|██████████| 1875/1875 [00:01<00:00, 1805.25it/s]\n",
      "100%|██████████| 1875/1875 [00:01<00:00, 1725.20it/s]\n",
      " 96%|█████████▌| 1802/1875 [00:01<00:00, 1978.26it/s]\n",
      "100%|██████████| 1875/1875 [00:01<00:00, 1795.67it/s]\n",
      "100%|██████████| 1875/1875 [00:01<00:00, 1627.62it/s]\n",
      "100%|██████████| 1875/1875 [00:01<00:00, 1812.63it/s]\n",
      "100%|██████████| 1875/1875 [00:00<00:00, 1895.44it/s]\n",
      "100%|██████████| 1875/1875 [00:01<00:00, 1845.50it/s]\n"
     ]
    }
   ],
   "source": [
    "results = random.sample(results, 30000)\n",
    "results1 = cleaning.multiprocessing(results, loop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 350928/350928 [00:29<00:00, 12000.80it/s]\n"
     ]
    }
   ],
   "source": [
    "with tf.io.gfile.GFile('segmentation-multisentences-news.tsv', \"w\") as outfile:\n",
    "    for i in tqdm(range(len(results1))):\n",
    "        l = cleaning_row(results1[i][0])\n",
    "        r = cleaning_row(results1[i][1])\n",
    "        outfile.write(\"%s\\t%s\\n\" % (l, r))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(files[2]) as fopen:\n",
    "    data = fopen.read().split('\\n')\n",
    "\n",
    "results, result = [], []\n",
    "for i in data:\n",
    "    if len(i) and i[-1] != '.':\n",
    "        i = i + '.'\n",
    "    if not len(i) and len(result):\n",
    "        results.append(result)\n",
    "        result = []\n",
    "    else:\n",
    "        if len(i):\n",
    "            result.append(i)\n",
    "        \n",
    "if len(result):\n",
    "    results.append(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 625/625 [00:00<00:00, 2700.11it/s]\n",
      "100%|██████████| 625/625 [00:00<00:00, 2868.21it/s]\n",
      "100%|██████████| 625/625 [00:00<00:00, 2989.80it/s]\n",
      "100%|██████████| 625/625 [00:00<00:00, 2495.67it/s]\n",
      "100%|██████████| 625/625 [00:00<00:00, 3054.79it/s]\n",
      "100%|██████████| 625/625 [00:00<00:00, 2787.96it/s]\n",
      "100%|██████████| 625/625 [00:00<00:00, 3051.07it/s]\n",
      "100%|██████████| 625/625 [00:00<00:00, 2906.34it/s]\n",
      "100%|██████████| 625/625 [00:00<00:00, 3010.58it/s]\n",
      "100%|██████████| 625/625 [00:00<00:00, 2926.35it/s]\n",
      "100%|██████████| 625/625 [00:00<00:00, 2352.78it/s]\n",
      "100%|██████████| 625/625 [00:00<00:00, 2644.63it/s]\n",
      "100%|██████████| 625/625 [00:00<00:00, 2277.55it/s]\n",
      "100%|██████████| 625/625 [00:00<00:00, 2935.51it/s]\n",
      "100%|██████████| 625/625 [00:00<00:00, 2713.23it/s]\n",
      "100%|██████████| 625/625 [00:00<00:00, 2288.38it/s]\n"
     ]
    }
   ],
   "source": [
    "results = random.sample(results, 10000)\n",
    "results1 = cleaning.multiprocessing(results, loop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119514/119514 [00:05<00:00, 21688.75it/s]\n"
     ]
    }
   ],
   "source": [
    "with tf.io.gfile.GFile('segmentation-multisentences-iium.tsv', \"w\") as outfile:\n",
    "    for i in tqdm(range(len(results1))):\n",
    "        l = cleaning_row(results1[i][0])\n",
    "        r = cleaning_row(results1[i][1])\n",
    "        outfile.write(\"%s\\t%s\\n\" % (l, r))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_short(string):\n",
    "    splitted = string.split()\n",
    "    random_length = random.randint(2, min(len(splitted), 10))\n",
    "    end = random.randint(0 + random_length, len(splitted))\n",
    "    return ' '.join(splitted[end - random_length: end])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2037249"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(files[0]) as fopen:\n",
    "    data = list(filter(None, fopen.read().split('\\n')))\n",
    "    \n",
    "data = [i for i in data if len(i) >= 2]\n",
    "\n",
    "len(data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Ogos 1983) merupakan pemain bola', 'Ogos 1983) merupakan pemain bola'),\n",
       " ('ogos 1983) merupakan pemain bola', 'ogos 1983) merupakan pemain bola'),\n",
       " ('Ogos1983) merupakanpemainbola', 'Ogos 1983) merupakan pemain bola'),\n",
       " ('ogos 1983)merupakan pemainbola', 'ogos 1983) merupakan pemain bola'),\n",
       " ('Ogos1983)merupakan pemainbola', 'Ogos 1983) merupakan pemain bola'),\n",
       " ('ogos1983) merupakanpemainbola', 'ogos 1983) merupakan pemain bola')]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "package(generate_short(data[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = random.sample(data, 500000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loop(strings):\n",
    "    results = []\n",
    "    for i in tqdm(range(len(strings))):\n",
    "        try:\n",
    "            p = package(generate_short(strings[i]))\n",
    "            results.extend(p)\n",
    "        except:\n",
    "            pass\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 31250/31250 [00:00<00:00, 31638.77it/s]\n",
      "100%|██████████| 31250/31250 [00:01<00:00, 27251.91it/s]\n",
      "100%|██████████| 31250/31250 [00:01<00:00, 27767.37it/s]\n",
      "100%|██████████| 31250/31250 [00:00<00:00, 31929.12it/s]\n",
      "100%|██████████| 31250/31250 [00:01<00:00, 26216.18it/s]\n",
      "100%|██████████| 31250/31250 [00:01<00:00, 27515.60it/s]\n",
      "100%|██████████| 31250/31250 [00:01<00:00, 28444.08it/s]\n",
      "100%|██████████| 31250/31250 [00:01<00:00, 26206.19it/s]\n",
      "100%|██████████| 31250/31250 [00:01<00:00, 29620.30it/s]\n",
      "100%|██████████| 31250/31250 [00:01<00:00, 27575.11it/s]\n",
      "100%|██████████| 31250/31250 [00:01<00:00, 26516.15it/s]\n",
      "100%|██████████| 31250/31250 [00:01<00:00, 27614.74it/s]\n",
      "100%|██████████| 31250/31250 [00:01<00:00, 26503.23it/s]\n",
      "100%|██████████| 31250/31250 [00:01<00:00, 25955.45it/s]\n",
      "100%|██████████| 31250/31250 [00:01<00:00, 26340.78it/s]\n",
      "100%|██████████| 31250/31250 [00:01<00:00, 25084.73it/s]\n"
     ]
    }
   ],
   "source": [
    "results1 = cleaning.multiprocessing(data, loop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Lapangan Terbang Stanley, Kanada.', 'Lapangan Terbang Stanley, Kanada.'),\n",
       " ('lapangan terbang stanley, kanada.', 'lapangan terbang stanley, kanada.'),\n",
       " ('LapanganTerbangStanley,Kanada.', 'Lapangan Terbang Stanley, Kanada.'),\n",
       " ('lapangan terbang stanley,kanada.', 'lapangan terbang stanley, kanada.'),\n",
       " ('LapanganTerbangStanley, Kanada.', 'Lapangan Terbang Stanley, Kanada.'),\n",
       " ('lapangan terbangstanley,kanada.', 'lapangan terbang stanley, kanada.'),\n",
       " ('ketara antara derma darah dan bekam itu sendiri iaitu',\n",
       "  'ketara antara derma darah dan bekam itu sendiri iaitu'),\n",
       " ('ketara antara derma darah dan bekam itu sendiri iaitu',\n",
       "  'ketara antara derma darah dan bekam itu sendiri iaitu'),\n",
       " ('ketaraantaraderma darahdanbekam itusendiriiaitu',\n",
       "  'ketara antara derma darah dan bekam itu sendiri iaitu'),\n",
       " ('ketaraantara dermadarahdan bekamitusendiriiaitu',\n",
       "  'ketara antara derma darah dan bekam itu sendiri iaitu')]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results1[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2985024/2985024 [00:32<00:00, 92321.84it/s] \n"
     ]
    }
   ],
   "source": [
    "with tf.io.gfile.GFile('segmentation-short-wiki.tsv', \"w\") as outfile:\n",
    "    for i in tqdm(range(len(results1))):\n",
    "        l = cleaning_row(results1[i][0])\n",
    "        r = cleaning_row(results1[i][1])\n",
    "        outfile.write(\"%s\\t%s\\n\" % (l, r))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(files[1]) as fopen:\n",
    "    data = list(filter(None, fopen.read().split('\\n')))\n",
    "    \n",
    "data = random.sample(data, 300000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 18750/18750 [00:00<00:00, 27738.48it/s]\n",
      "100%|██████████| 18750/18750 [00:00<00:00, 27987.66it/s]\n",
      "100%|██████████| 18750/18750 [00:00<00:00, 24858.23it/s]\n",
      "100%|██████████| 18750/18750 [00:00<00:00, 24817.68it/s]\n",
      "100%|██████████| 18750/18750 [00:00<00:00, 25724.49it/s]\n",
      "100%|██████████| 18750/18750 [00:00<00:00, 24575.50it/s]\n",
      "100%|██████████| 18750/18750 [00:00<00:00, 22110.61it/s]\n",
      "100%|██████████| 18750/18750 [00:00<00:00, 22701.33it/s]\n",
      "100%|██████████| 18750/18750 [00:00<00:00, 24747.30it/s]\n",
      "100%|██████████| 18750/18750 [00:00<00:00, 24896.39it/s]\n",
      "100%|██████████| 18750/18750 [00:00<00:00, 22432.08it/s]\n",
      "100%|██████████| 18750/18750 [00:00<00:00, 23395.33it/s]\n",
      "100%|██████████| 18750/18750 [00:00<00:00, 23290.79it/s]\n",
      "100%|██████████| 18750/18750 [00:00<00:00, 19031.38it/s]\n",
      "100%|██████████| 18750/18750 [00:00<00:00, 20608.64it/s]\n",
      "100%|██████████| 18750/18750 [00:00<00:00, 20800.25it/s]\n"
     ]
    }
   ],
   "source": [
    "results1 = cleaning.multiprocessing(data, loop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1786584/1786584 [00:19<00:00, 90023.01it/s] \n"
     ]
    }
   ],
   "source": [
    "with tf.io.gfile.GFile('segmentation-short-news.tsv', \"w\") as outfile:\n",
    "    for i in tqdm(range(len(results1))):\n",
    "        l = cleaning_row(results1[i][0])\n",
    "        r = cleaning_row(results1[i][1])\n",
    "        outfile.write(\"%s\\t%s\\n\" % (l, r))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1121978"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(files[2]) as fopen:\n",
    "    data = list(filter(None, fopen.read().split('\\n')))\n",
    "    \n",
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = random.sample(data, 200000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 12500/12500 [00:00<00:00, 28058.40it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 29924.00it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 33892.53it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 38696.78it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 27346.08it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 27567.42it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 36103.69it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 25157.28it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 26097.73it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 25318.00it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 28291.60it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 31994.55it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 25025.31it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 30093.43it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 29402.30it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 26298.85it/s]\n"
     ]
    }
   ],
   "source": [
    "results1 = cleaning.multiprocessing(data, loop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1199244/1199244 [00:11<00:00, 100127.38it/s]\n"
     ]
    }
   ],
   "source": [
    "with tf.io.gfile.GFile('segmentation-short-iium.tsv', \"w\") as outfile:\n",
    "    for i in tqdm(range(len(results1))):\n",
    "        l = cleaning_row(results1[i][0])\n",
    "        r = cleaning_row(results1[i][1])\n",
    "        outfile.write(\"%s\\t%s\\n\" % (l, r))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(files[0]) as fopen:\n",
    "    data = list(filter(None, fopen.read().split('\\n')))\n",
    "    \n",
    "data = [i for i in data if len(i) >= 2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loop(strings):\n",
    "    results = []\n",
    "    for i in tqdm(range(len(strings))):\n",
    "        p = package(strings[i])\n",
    "        results.extend(p)\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 18750/18750 [00:01<00:00, 16996.60it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 16581.23it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 14867.02it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 15988.75it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 16454.46it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 14050.37it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 14639.87it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 15140.95it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 14234.41it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 14246.56it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 13768.30it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 13521.40it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 13621.37it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 14762.65it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 14001.68it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 13679.29it/s]\n"
     ]
    }
   ],
   "source": [
    "data = random.sample(data, 300000)\n",
    "results1 = cleaning.multiprocessing(data, loop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1800000/1800000 [00:34<00:00, 52350.61it/s]\n"
     ]
    }
   ],
   "source": [
    "with tf.io.gfile.GFile('segmentation-wiki.tsv', \"w\") as outfile:\n",
    "    for i in tqdm(range(len(results1))):\n",
    "        l = cleaning_row(results1[i][0])\n",
    "        r = cleaning_row(results1[i][1])\n",
    "        outfile.write(\"%s\\t%s\\n\" % (l, r))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 18750/18750 [00:01<00:00, 11662.35it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 13202.20it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 13002.63it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 13574.42it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 13590.34it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 13395.15it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 12538.18it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 13064.20it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 12705.65it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 12072.20it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 12228.56it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 12219.75it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 11435.33it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 11102.09it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 10995.65it/s]\n",
      "100%|██████████| 18750/18750 [00:01<00:00, 10824.40it/s]\n"
     ]
    }
   ],
   "source": [
    "with open(files[1]) as fopen:\n",
    "    data = list(filter(None, fopen.read().split('\\n')))\n",
    "    \n",
    "data = random.sample(data, 300000)\n",
    "results1 = cleaning.multiprocessing(data, loop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1800000/1800000 [00:38<00:00, 46487.60it/s]\n"
     ]
    }
   ],
   "source": [
    "with tf.io.gfile.GFile('segmentation-news.tsv', \"w\") as outfile:\n",
    "    for i in tqdm(range(len(results1))):\n",
    "        l = cleaning_row(results1[i][0])\n",
    "        r = cleaning_row(results1[i][1])\n",
    "        outfile.write(\"%s\\t%s\\n\" % (l, r))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1121978"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(files[2]) as fopen:\n",
    "    data = list(filter(None, fopen.read().split('\\n')))\n",
    "    \n",
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = random.sample(data, 200000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 12500/12500 [00:00<00:00, 22022.69it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 21196.58it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 21614.38it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 20727.71it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 20952.80it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 20567.83it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 20907.93it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 19533.26it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 20796.93it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 19171.59it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 18818.06it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 17295.04it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 17200.69it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 18796.54it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 18671.64it/s]\n",
      "100%|██████████| 12500/12500 [00:00<00:00, 16281.22it/s]\n"
     ]
    }
   ],
   "source": [
    "results1 = cleaning.multiprocessing(data, loop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1200000/1200000 [00:17<00:00, 67248.05it/s]\n"
     ]
    }
   ],
   "source": [
    "with tf.io.gfile.GFile('segmentation-iium.tsv', \"w\") as outfile:\n",
    "    for i in tqdm(range(len(results1))):\n",
    "        l = cleaning_row(results1[i][0])\n",
    "        r = cleaning_row(results1[i][1])\n",
    "        outfile.write(\"%s\\t%s\\n\" % (l, r))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'mesolitica-tpu.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import storage\n",
    "\n",
    "client = storage.Client()\n",
    "bucket = client.bucket('mesolitica-tpu-general')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['segmentation-short-wiki.tsv',\n",
       " 'segmentation-short-iium.tsv',\n",
       " 'segmentation-multisentences-wiki.tsv',\n",
       " 'segmentation-short-news.tsv',\n",
       " 'segmentation-news.tsv',\n",
       " 'segmentation-multisentences-iium.tsv',\n",
       " 'segmentation-iium.tsv',\n",
       " 'segmentation-wiki.tsv',\n",
       " 'segmentation-multisentences-news.tsv']"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from glob import glob\n",
    "\n",
    "files = glob('segmentation*.tsv')\n",
    "files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "segmentation-short-wiki.tsv\n",
      "segmentation-short-iium.tsv\n",
      "segmentation-multisentences-wiki.tsv\n",
      "segmentation-short-news.tsv\n",
      "segmentation-news.tsv\n",
      "segmentation-multisentences-iium.tsv\n",
      "segmentation-iium.tsv\n",
      "segmentation-wiki.tsv\n",
      "segmentation-multisentences-news.tsv\n"
     ]
    }
   ],
   "source": [
    "for file in files:\n",
    "    print(file)\n",
    "    blob = bucket.blob(f't5-data-v2/{file}')\n",
    "    blob.upload_from_filename(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "b2_application_key_id = os.environ['b2_application_key_id']\n",
    "b2_application_key = os.environ['b2_application_key']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from b2sdk.v1 import *\n",
    "info = InMemoryAccountInfo()\n",
    "b2_api = B2Api(info)\n",
    "application_key_id = b2_application_key_id\n",
    "application_key = b2_application_key\n",
    "b2_api.authorize_account(\"production\", application_key_id, application_key)\n",
    "file_info = {'how': 'good-file'}\n",
    "b2_bucket = b2_api.get_bucket_by_name('malay-dataset')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "segmentation-short-wiki.tsv\n",
      "segmentation-short-iium.tsv\n",
      "segmentation-multisentences-wiki.tsv\n",
      "segmentation-short-news.tsv\n",
      "segmentation-news.tsv\n",
      "segmentation-multisentences-iium.tsv\n",
      "segmentation-iium.tsv\n",
      "segmentation-wiki.tsv\n",
      "segmentation-multisentences-news.tsv\n"
     ]
    }
   ],
   "source": [
    "for file in files:\n",
    "    print(file)\n",
    "    b2_bucket.upload_local_file(\n",
    "    local_file=file,\n",
    "    file_name=f'segmentation/{file}',\n",
    "    file_infos=file_info,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "for file in files:\n",
    "    os.remove(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_datasets as tfds\n",
    "from t5.data import preprocessors as prep\n",
    "import functools\n",
    "import t5\n",
    "import gin\n",
    "import sentencepiece as spm\n",
    "from glob import glob\n",
    "import os\n",
    "\n",
    "gin.parse_config_file('pretrained_models_base_operative_config.gin')\n",
    "vocab = 'sp10m.cased.ms-en.model'\n",
    "sp = spm.SentencePieceProcessor()\n",
    "sp.Load(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def segmentation_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(\n",
    "        [\n",
    "            'segmentation-short-wiki.tsv'\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['question', 'answer'], ex)))\n",
    "    return ds\n",
    "\n",
    "def segmentation_preprocessor(ds):\n",
    "    def to_inputs_and_targets(ex):\n",
    "        return {\n",
    "            'inputs': tf.strings.join(['segmentasi: ', ex['question']]),\n",
    "            'targets': ex['answer'],\n",
    "        }\n",
    "\n",
    "    return ds.map(\n",
    "        to_inputs_and_targets,\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "t5.data.TaskRegistry.remove('segmentation_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'segmentation_dataset',\n",
    "    dataset_fn = segmentation_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = [segmentation_preprocessor],\n",
    "    sentencepiece_model_path = vocab,\n",
    "    metric_fns = [t5.evaluation.metrics.accuracy],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "nq_task = t5.data.TaskRegistry.get(\"segmentation_dataset\")\n",
    "ds = nq_task.get_dataset(split='knowledge-graph.tsv', sequence_length={\"inputs\": 256, \"targets\": 256})\n",
    "r = tfds.as_numpy(ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "ename": "NotFoundError",
     "evalue": "segmentation-short-wiki.tsv; No such file or directory\n\t [[node IteratorGetNext (defined at /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py:1748) ]]\n\nOriginal stack trace for 'IteratorGetNext':\n  File \"/usr/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/usr/lib/python3.6/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/home/husein/.local/lib/python3.6/site-packages/ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"/home/husein/.local/lib/python3.6/site-packages/traitlets/config/application.py\", line 664, in launch_instance\n    app.start()\n  File \"/home/husein/.local/lib/python3.6/site-packages/ipykernel/kernelapp.py\", line 563, in start\n    self.io_loop.start()\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/platform/asyncio.py\", line 148, in start\n    self.asyncio_loop.run_forever()\n  File \"/usr/lib/python3.6/asyncio/base_events.py\", line 438, in run_forever\n    self._run_once()\n  File \"/usr/lib/python3.6/asyncio/base_events.py\", line 1451, in _run_once\n    handle._run()\n  File \"/usr/lib/python3.6/asyncio/events.py\", line 145, in _run\n    self._callback(*self._args)\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/ioloop.py\", line 690, in <lambda>\n    lambda f: self._run_callback(functools.partial(callback, future))\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/ioloop.py\", line 743, in _run_callback\n    ret = callback()\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/gen.py\", line 787, in inner\n    self.run()\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/gen.py\", line 748, in run\n    yielded = self.gen.send(value)\n  File \"/home/husein/.local/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 378, in dispatch_queue\n    yield self.process_one()\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/gen.py\", line 225, in wrapper\n    runner = Runner(result, future, yielded)\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/gen.py\", line 714, in __init__\n    self.run()\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/gen.py\", line 748, in run\n    yielded = self.gen.send(value)\n  File \"/home/husein/.local/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 365, in process_one\n    yield gen.maybe_future(dispatch(*args))\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"/home/husein/.local/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 272, in dispatch_shell\n    yield gen.maybe_future(handler(stream, idents, msg))\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"/home/husein/.local/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 542, in execute_request\n    user_expressions, allow_stdin,\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"/home/husein/.local/lib/python3.6/site-packages/ipykernel/ipkernel.py\", line 294, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/home/husein/.local/lib/python3.6/site-packages/ipykernel/zmqshell.py\", line 536, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/home/husein/.local/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2855, in run_cell\n    raw_cell, store_history, silent, shell_futures)\n  File \"/home/husein/.local/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2881, in _run_cell\n    return runner(coro)\n  File \"/home/husein/.local/lib/python3.6/site-packages/IPython/core/async_helpers.py\", line 68, in _pseudo_sync_runner\n    coro.send(None)\n  File \"/home/husein/.local/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3058, in run_cell_async\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/home/husein/.local/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3249, in run_ast_nodes\n    if (await self.run_code(code, result,  async_=asy)):\n  File \"/home/husein/.local/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3326, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-53-8ebe59a56b1d>\", line 1, in <module>\n    next(r)\n  File \"/home/husein/.local/lib/python3.6/site-packages/tensorflow_datasets/core/dataset_utils.py\", line 171, in _graph_dataset_iterator\n    ds_item = ds_iter.get_next()\n  File \"/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/data/ops/iterator_ops.py\", line 426, in get_next\n    name=name)\n  File \"/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/gen_dataset_ops.py\", line 2518, in iterator_get_next\n    output_shapes=output_shapes, name=name)\n  File \"/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/op_def_library.py\", line 794, in _apply_op_helper\n    op_def=op_def)\n  File \"/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py\", line 507, in new_func\n    return func(*args, **kwargs)\n  File \"/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py\", line 3357, in create_op\n    attrs, op_def, compute_device)\n  File \"/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py\", line 3426, in _create_op_internal\n    op_def=op_def)\n  File \"/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py\", line 1748, in __init__\n    self._traceback = tf_stack.extract_stack()\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNotFoundError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1364\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1365\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1366\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1349\u001b[0m       return self._call_tf_sessionrun(options, feed_dict, fetch_list,\n\u001b[0;32m-> 1350\u001b[0;31m                                       target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1351\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1442\u001b[0m                                             \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1443\u001b[0;31m                                             run_metadata)\n\u001b[0m\u001b[1;32m   1444\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNotFoundError\u001b[0m: segmentation-short-wiki.tsv; No such file or directory\n\t [[{{node IteratorGetNext}}]]",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mNotFoundError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-53-8ebe59a56b1d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_datasets/core/dataset_utils.py\u001b[0m in \u001b[0;36m_graph_dataset_iterator\u001b[0;34m(ds_iter, graph)\u001b[0m\n\u001b[1;32m    174\u001b[0m     \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    175\u001b[0m       \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 176\u001b[0;31m         \u001b[0;32myield\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mds_item\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    177\u001b[0m       \u001b[0;32mexcept\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOutOfRangeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    178\u001b[0m         \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    954\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    955\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 956\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    957\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    958\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1178\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1179\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1180\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1181\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1182\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1357\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1358\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1359\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1360\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1361\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1382\u001b[0m                     \u001b[0;34m'\\nsession_config.graph_options.rewrite_options.'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1383\u001b[0m                     'disable_meta_optimizer = True')\n\u001b[0;32m-> 1384\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1385\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1386\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNotFoundError\u001b[0m: segmentation-short-wiki.tsv; No such file or directory\n\t [[node IteratorGetNext (defined at /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py:1748) ]]\n\nOriginal stack trace for 'IteratorGetNext':\n  File \"/usr/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/usr/lib/python3.6/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/home/husein/.local/lib/python3.6/site-packages/ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"/home/husein/.local/lib/python3.6/site-packages/traitlets/config/application.py\", line 664, in launch_instance\n    app.start()\n  File \"/home/husein/.local/lib/python3.6/site-packages/ipykernel/kernelapp.py\", line 563, in start\n    self.io_loop.start()\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/platform/asyncio.py\", line 148, in start\n    self.asyncio_loop.run_forever()\n  File \"/usr/lib/python3.6/asyncio/base_events.py\", line 438, in run_forever\n    self._run_once()\n  File \"/usr/lib/python3.6/asyncio/base_events.py\", line 1451, in _run_once\n    handle._run()\n  File \"/usr/lib/python3.6/asyncio/events.py\", line 145, in _run\n    self._callback(*self._args)\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/ioloop.py\", line 690, in <lambda>\n    lambda f: self._run_callback(functools.partial(callback, future))\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/ioloop.py\", line 743, in _run_callback\n    ret = callback()\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/gen.py\", line 787, in inner\n    self.run()\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/gen.py\", line 748, in run\n    yielded = self.gen.send(value)\n  File \"/home/husein/.local/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 378, in dispatch_queue\n    yield self.process_one()\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/gen.py\", line 225, in wrapper\n    runner = Runner(result, future, yielded)\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/gen.py\", line 714, in __init__\n    self.run()\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/gen.py\", line 748, in run\n    yielded = self.gen.send(value)\n  File \"/home/husein/.local/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 365, in process_one\n    yield gen.maybe_future(dispatch(*args))\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"/home/husein/.local/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 272, in dispatch_shell\n    yield gen.maybe_future(handler(stream, idents, msg))\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"/home/husein/.local/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 542, in execute_request\n    user_expressions, allow_stdin,\n  File \"/home/husein/.local/lib/python3.6/site-packages/tornado/gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"/home/husein/.local/lib/python3.6/site-packages/ipykernel/ipkernel.py\", line 294, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/home/husein/.local/lib/python3.6/site-packages/ipykernel/zmqshell.py\", line 536, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/home/husein/.local/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2855, in run_cell\n    raw_cell, store_history, silent, shell_futures)\n  File \"/home/husein/.local/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2881, in _run_cell\n    return runner(coro)\n  File \"/home/husein/.local/lib/python3.6/site-packages/IPython/core/async_helpers.py\", line 68, in _pseudo_sync_runner\n    coro.send(None)\n  File \"/home/husein/.local/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3058, in run_cell_async\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/home/husein/.local/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3249, in run_ast_nodes\n    if (await self.run_code(code, result,  async_=asy)):\n  File \"/home/husein/.local/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3326, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-53-8ebe59a56b1d>\", line 1, in <module>\n    next(r)\n  File \"/home/husein/.local/lib/python3.6/site-packages/tensorflow_datasets/core/dataset_utils.py\", line 171, in _graph_dataset_iterator\n    ds_item = ds_iter.get_next()\n  File \"/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/data/ops/iterator_ops.py\", line 426, in get_next\n    name=name)\n  File \"/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/gen_dataset_ops.py\", line 2518, in iterator_get_next\n    output_shapes=output_shapes, name=name)\n  File \"/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/op_def_library.py\", line 794, in _apply_op_helper\n    op_def=op_def)\n  File \"/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py\", line 507, in new_func\n    return func(*args, **kwargs)\n  File \"/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py\", line 3357, in create_op\n    attrs, op_def, compute_device)\n  File \"/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py\", line 3426, in _create_op_internal\n    op_def=op_def)\n  File \"/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py\", line 1748, in __init__\n    self._traceback = tf_stack.extract_stack()\n"
     ]
    }
   ],
   "source": [
    "next(r)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
